{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Knet CNN Example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "true"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# using Pkg; pkg\"add Knet MLDatasets\"\n",
    "using Knet, MLDatasets\n",
    "True=true # so we can read the python params\n",
    "include(\"common/params.py\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OS: Linux\n",
      "Julia: 1.5.0\n",
      "Knet: 1.4.0\n",
      "GPU: GeForce GTX 1060 with Max-Q Design, \n"
     ]
    }
   ],
   "source": [
    "println(\"OS: \", Sys.KERNEL)\n",
    "println(\"Julia: \", VERSION)\n",
    "println(\"Knet: \", Pkg.dependencies()[Base.UUID(\"1902f260-5fb4-5aff-8c31-6271790ab950\")].version)\n",
    "println(\"GPU: \", replace(read(`nvidia-smi --query-gpu=name --format=csv,noheader`,String),'\\n'=>\", \"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define model\n",
    "function initmodel(; atype=KnetArray, dtype=Float32, winit=xavier, binit=zeros)\n",
    "    w(dims...)=atype(winit(dtype,dims...))\n",
    "    b(dims...)=atype(binit(dtype,dims...))\n",
    "    return Any[\n",
    "        w(3,3,3,50), b(1,1,50,1),\n",
    "        w(3,3,50,50), b(1,1,50,1),\n",
    "        w(3,3,50,100), b(1,1,100,1),\n",
    "        w(3,3,100,100), b(1,1,100,1),\n",
    "        w(512,6400), b(512,1),\n",
    "        w(10,512), b(10,1)\n",
    "    ]\n",
    "end;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define loss and its gradient\n",
    "function predict(w,x; pdrop=(0,0))\n",
    "    convbias(x,w,b) = conv4(w,x;padding=1) .+ b\n",
    "    fc(x,w,b) = w * mat(x) .+ b;\n",
    "    x = relu.(convbias(x,w[1],w[2]))\n",
    "    x = relu.(pool(convbias(x,w[3],w[4])))\n",
    "    x = dropout(x,pdrop[1])\n",
    "    x = relu.(convbias(x,w[5],w[6]))\n",
    "    x = relu.(pool(convbias(x,w[7],w[8])))\n",
    "    x = dropout(x,pdrop[1])\n",
    "    x = relu.(fc(x,w[9],w[10]))\n",
    "    x = dropout(x,pdrop[2])\n",
    "    return fc(x,w[11],w[12])\n",
    "end\n",
    "\n",
    "loss(w,x,y;o...)=nll(predict(w,x;o...),y) # nll: negative log likelihood\n",
    "lossgradient = grad(loss);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "32×32×3×50000 KnetArray{Float32,4}\n",
      "50000-element Array{Int64,1}\n",
      "32×32×3×10000 KnetArray{Float32,4}\n",
      "10000-element Array{Int64,1}\n"
     ]
    }
   ],
   "source": [
    "# load data\n",
    "xtrn, ytrn = CIFAR10.traindata(); xtrn = convert(KnetArray{Float32},xtrn)\n",
    "xtst, ytst = CIFAR10.testdata(); xtst = convert(KnetArray{Float32},xtst)\n",
    "dtrn = minibatch(xtrn,ytrn,BATCHSIZE,shuffle=true,xtype=KnetArray{Float32})\n",
    "dtst = minibatch(xtst,ytst,BATCHSIZE,shuffle=false,xtype=KnetArray{Float32})\n",
    "for d in (xtrn,ytrn,xtst,ytst); println(summary(d)); end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "┌ Warning: optimizers is deprecated, use sgd, adam etc. instead.\n",
      "└ @ Knet.Train20 /home/deniz/.julia/dev/Knet/src/train20/update.jl:598\n"
     ]
    }
   ],
   "source": [
    "# prepare for training\n",
    "model = optim = nothing; GC.gc(true) # Clear memory from last run\n",
    "model = initmodel()\n",
    "optim = optimizers(model, Momentum; lr=LR, gamma=MOMENTUM);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 22.001135 seconds (39.44 M allocations: 1.999 GiB, 2.86% gc time)\n"
     ]
    }
   ],
   "source": [
    "# cold start\n",
    "@time for (x,y) in dtrn\n",
    "    grads = lossgradient(model, x, y; pdrop=(0.25,0.5))\n",
    "    update!(model, grads, optim)\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# prepare for training\n",
    "model = optim = nothing; GC.gc(true) # Clear memory from last run\n",
    "model = initmodel()\n",
    "optim = optimizers(model, Momentum; lr=LR, gamma=MOMENTUM);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "┌ Info: Training...\n",
      "└ @ Main In[9]:2\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  9.981000 seconds (3.40 M allocations: 180.850 MiB, 1.54% gc time)\n",
      " 10.001566 seconds (3.40 M allocations: 180.621 MiB, 1.52% gc time)\n",
      " 10.040050 seconds (3.40 M allocations: 180.862 MiB, 1.55% gc time)\n",
      " 10.338632 seconds (3.40 M allocations: 180.798 MiB, 1.63% gc time)\n",
      " 10.792743 seconds (3.40 M allocations: 180.749 MiB, 1.73% gc time)\n",
      " 10.497737 seconds (3.40 M allocations: 180.864 MiB, 1.68% gc time)\n",
      " 10.485780 seconds (3.40 M allocations: 180.810 MiB, 1.67% gc time)\n",
      " 10.392641 seconds (3.40 M allocations: 180.971 MiB, 1.68% gc time)\n",
      " 10.274777 seconds (3.40 M allocations: 180.951 MiB, 1.69% gc time)\n",
      " 10.284816 seconds (3.40 M allocations: 180.708 MiB, 1.72% gc time)\n",
      "103.091919 seconds (34.05 M allocations: 1.766 GiB, 1.64% gc time)\n"
     ]
    }
   ],
   "source": [
    "# 159s\n",
    "@info(\"Training...\")\n",
    "@time for epoch in 1:EPOCHS\n",
    "    @time for (x,y) in dtrn\n",
    "        grads = lossgradient(model, x, y; pdrop=(0.25,0.5))\n",
    "        update!(model, grads, optim)\n",
    "    end\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "┌ Warning: accuracy(weights,data,func; o...) is deprecated, please use accuracy(x->func(weights,x); data=data, o...) instead.\n",
      "└ @ Knet.Ops20 /home/deniz/.julia/dev/Knet/src/ops20/loss.jl:227\n",
      "┌ Warning: accuracy(model,data; o...) is deprecated, please use accuracy(model; data=data, o...)\n",
      "└ @ Knet.Ops20 /home/deniz/.julia/dev/Knet/src/ops20/loss.jl:205\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  1.908039 seconds (3.76 M allocations: 184.310 MiB, 2.54% gc time)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.7825506343200535"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# test accuracy 77.54\n",
    "@time accuracy(model,dtst,predict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Julia 1.5.0",
   "language": "julia",
   "name": "julia-1.5"
  },
  "language_info": {
   "file_extension": ".jl",
   "mimetype": "application/julia",
   "name": "julia",
   "version": "1.5.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
